// coding: utf-8
// @Author : lryself
// @Date : 2021/2/27 17:50
// @Software: GoLand

package main

import (
	"bufio"
	"fmt"
	"github.com/PuerkitoBio/goquery"
	log "github.com/sirupsen/logrus"
	"io"
	"net/http"
	"os"
	"path"
	"strings"
	"sync"
	"time"
)

var wg sync.WaitGroup

func init() {
	// 设置日志格式为json格式
	log.SetFormatter(&log.JSONFormatter{})

	// 设置将日志输出到标准输出（默认的输出为stderr，标准错误）
	// 日志消息输出可以是任意的io.writer类型
	log.SetOutput(os.Stdout)

	// 设置日志级别为warn以上
	log.SetLevel(log.WarnLevel)
}

func main() {
	start := time.Now()
	dirPath,err :=os.Getwd()
	if err != nil {
		panic(err)
	}
	imgPath := dirPath + "\\download\\"
	log.Infoln("图片储存位置：", imgPath)

	for page:=1;page<=30;page++ {
		log.Infoln("开始爬取第%d页\n", page)
		downloadImagePage(
			fmt.Sprintf("https://wallhaven.cc/search?categories=100&purity=100&ratios=16x9&topRange=1M&sorting=toplist&order=desc&page=%d", page),
			imgPath)
	}
	wg.Wait()
	//downloadImage("https://w.wallhaven.cc/full/y8/wallhaven-y88j9g.jpg")

	log.Infoln("共用时：",time.Since(start).Seconds(),"秒")
	fmt.Print("输入回车键结束程序！")
	_,_ =fmt.Scanf("%s")
}

func downloadImagePage(pageUrl string, imagePath string) {
	imgUrls := parsePage(pageUrl)
	for _, imgUrl := range imgUrls {
		wg.Add(1)
		go downloadImage(imgUrl, imagePath)
	}
	wg.Wait()
}
func parsePage(pageUrl string) []string{
	response,err := http.Get(pageUrl)
	if err != nil {
		log.Println(err)
		return nil
	}
	defer response.Body.Close()

	if response.StatusCode != 200 {
		log.Errorln("status code error: %d %s", response.StatusCode, response.Status)
		return nil
	}

	// 创建解析器
	doc, err := goquery.NewDocumentFromReader(response.Body)
	if err != nil {
		log.Errorln(err)
		return nil
	}
	var imageUrls []string
	doc.Find("#thumbs > section:nth-child(1) > ul").Find("img").Each(func(i int, selection *goquery.Selection) {
		tempUrl,_ := selection.Attr("data-src")
		imageUrls = append(imageUrls, rePictureUrl(tempUrl))
	})

	return imageUrls
}

func rePictureUrl(oldUrl string) string {
	pictureName := path.Base(oldUrl)
	newUrl := strings.Replace(oldUrl, "https://th.wallhaven.cc/small", "https://w.wallhaven.cc/full", 1)
	newUrl = strings.Replace(newUrl, pictureName, "wallhaven-"+pictureName, 1)
	return newUrl
}

func downloadImage(imgUrl string, imgPath string) {
	defer wg.Done()
	//imgPath := "F:\\programme\\go\\go_learning\\src\\others\\walhaven\\download\\"
	imageName := path.Base(imgUrl)
	log.Infoln(imageName,"——开始下载")
	response,err := http.Get(imgUrl)
	if err != nil{
		fmt.Println(err)
		return
	}
	defer response.Body.Close()

	log.Infoln(imageName, "——下载完成！开始写入")

	f, err := os.Create(imgPath + imageName)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	reader := bufio.NewReaderSize(response.Body, 10 * 1024 * 1024)

	// 获得文件的writer对象
	writer := bufio.NewWriterSize(f, int(response.ContentLength))

	//start := time.Now()
	written, err := io.CopyBuffer(writer, reader, make([]byte, 10 * 1024 * 1024))
	//log.Println(time.Since(start).Seconds())

	//written, err := reader.WriteTo(writer)
	//if err != nil{
	//	fmt.Println(imageName, err)
	//}

	//bytes := make([]byte, response.ContentLength)
	//written := 0
	//for {
	//	ln, err := response.Body.Read(bytes)
	//
	//	if ln < 0 || err != nil{
	//		break
	//	}
	//	// 注意这里byte数组后的[0:ln]，不然可能会导致写入多余的数据
	//	nw, err := writer.Write(bytes[0:ln])
	//	written += nw
	//	if err != nil{
	//		fmt.Println(imageName, err)
	//	}
	//	fmt.Println(written*100/int(response.ContentLength), "%")
	//}

	//fmt.Println(time.Since(start).Seconds())
	log.Infof("%s——写入完成，大小： %dKB\n", imageName, written/1024)
}